// ==============================================================================================
// ╦  ┬┌─┐┌─┐┌┐┌┌─┐┌─┐    Yaksha Programming Language
// ║  ││  ├┤ │││└─┐├┤     is Licensed with GPLv3 + extra terms. Please see below.
// ╩═╝┴└─┘└─┘┘└┘└─┘└─┘
// Note: libs - MIT license, runtime/3rd - various
// ==============================================================================================
// GPLv3:
//
// Yaksha - Programming Language.
// Copyright (C) 2020 - 2024 Bhathiya Perera
//
// This program is free software: you can redistribute it and/or modify it under the terms
// of the GNU General Public License as published by the Free Software Foundation,
// either version 3 of the License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
// or FITNESS FOR A PARTICULAR PURPOSE.
// See the GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License along with this program.
// If not, see https://www.gnu.org/licenses/.
//
// ==============================================================================================
// Additional Terms:
//
// Please note that any commercial use of the programming language's compiler source code
// (everything except compiler/runtime, compiler/libs and compiler/3rd) require a written agreement
// with author of the language (Bhathiya Perera).
//
// If you are using it for an open source project, please give credits.
// Your own project must use GPLv3 license with these additional terms.
//
// You may use programs written in Yaksha/YakshaLisp for any legal purpose
// (commercial, open-source, closed-source, etc) as long as it agrees
// to the licenses of linked runtime libraries (see compiler/runtime/README.md).
//
// ==============================================================================================
// token.h
// generated by update_tokens.py
#ifndef TOKEN_H
#define TOKEN_H
#include <filesystem>
#include <string>
#include <vector>
namespace yaksha {
  /**
 * Most simple types of tokens we can match with tokenizing
 */
  enum class token_type {
    AND,
    AND_EQ,
    ARROW,
    AT,
    BA_DEDENT,
    BA_INDENT,
    COLON,
    COMMA,
    COMMENT,
    CURLY_BRACKET_CLOSE,
    CURLY_BRACKET_OPEN,
    DIV,
    DIV_EQ,
    DOT,
    DOUBLE_NUMBER,
    ELLIPSIS,
    END_OF_FILE,
    EQ,
    EQ_EQ,
    FLOAT_NUMBER,
    GREAT,
    GREAT_EQ,
    INDENT,
    INTEGER_BIN,
    INTEGER_BIN_16,
    INTEGER_BIN_64,
    INTEGER_BIN_8,
    INTEGER_DECIMAL,
    INTEGER_DECIMAL_16,
    INTEGER_DECIMAL_64,
    INTEGER_DECIMAL_8,
    INTEGER_HEX,
    INTEGER_HEX_16,
    INTEGER_HEX_64,
    INTEGER_HEX_8,
    INTEGER_OCT,
    INTEGER_OCT_16,
    INTEGER_OCT_64,
    INTEGER_OCT_8,
    INT_DIV,
    INT_DIV_EQ,
    LESS,
    LESS_EQ,
    MOD,
    MOD_EQ,
    MUL,
    MUL_EQ,
    NAME,
    NEW_LINE,
    NOT_EQ,
    NOT_SYMBOL,
    OR,
    OR_EQ,
    PAREN_CLOSE,
    PAREN_OPEN,
    PLUS,
    PLUS_EQ,
    POWER,
    POWER_EQ,
    SEMICOLON,
    SHL,
    SHL_EQ,
    SHR,
    SHR_EQ,
    SQUARE_BRACKET_CLOSE,
    SQUARE_BRACKET_OPEN,
    STRING,
    SUB,
    SUB_EQ,
    THREE_QUOTE_STRING,
    TILDE,
    UINTEGER_BIN,
    UINTEGER_BIN_16,
    UINTEGER_BIN_64,
    UINTEGER_BIN_8,
    UINTEGER_DECIMAL,
    UINTEGER_DECIMAL_16,
    UINTEGER_DECIMAL_64,
    UINTEGER_DECIMAL_8,
    UINTEGER_HEX,
    UINTEGER_HEX_16,
    UINTEGER_HEX_64,
    UINTEGER_HEX_8,
    UINTEGER_OCT,
    UINTEGER_OCT_16,
    UINTEGER_OCT_64,
    UINTEGER_OCT_8,
    UNKNOWN_BIN,
    UNKNOWN_DECIMAL,
    UNKNOWN_HEX,
    UNKNOWN_OCT,
    XOR,
    XOR_EQ,
    KEYWORD_FALSE,
    KEYWORD_NONE,
    KEYWORD_TRUE,
    KEYWORD_AND,
    KEYWORD_AS,
    KEYWORD_ASSERT,
    KEYWORD_BREAK,
    KEYWORD_CCODE,
    KEYWORD_CLASS,
    KEYWORD_CONTINUE,
    KEYWORD_DEF,
    KEYWORD_DEFER,
    KEYWORD_DEL,
    KEYWORD_DIRECTIVE,
    KEYWORD_ELIF,
    KEYWORD_ELSE,
    KEYWORD_FOR,
    KEYWORD_FROM,
    KEYWORD_IF,
    KEYWORD_IMPORT,
    KEYWORD_IN,
    KEYWORD_MACROS,
    KEYWORD_NOT,
    KEYWORD_OR,
    KEYWORD_PASS,
    KEYWORD_RETURN,
    KEYWORD_RUNTIMEFEATURE,
    KEYWORD_STRUCT,
    KEYWORD_TRY,
    KEYWORD_WHILE,
    TK_UNKNOWN_TOKEN_DETECTED
  };
  static inline bool is_keyword(const token_type &t) {
    bool is_kw = false;
    switch (t) {
      case token_type::KEYWORD_FALSE:
      case token_type::KEYWORD_NONE:
      case token_type::KEYWORD_TRUE:
      case token_type::KEYWORD_AND:
      case token_type::KEYWORD_AS:
      case token_type::KEYWORD_ASSERT:
      case token_type::KEYWORD_BREAK:
      case token_type::KEYWORD_CCODE:
      case token_type::KEYWORD_CLASS:
      case token_type::KEYWORD_CONTINUE:
      case token_type::KEYWORD_DEF:
      case token_type::KEYWORD_DEFER:
      case token_type::KEYWORD_DEL:
      case token_type::KEYWORD_DIRECTIVE:
      case token_type::KEYWORD_ELIF:
      case token_type::KEYWORD_ELSE:
      case token_type::KEYWORD_FOR:
      case token_type::KEYWORD_FROM:
      case token_type::KEYWORD_IF:
      case token_type::KEYWORD_IMPORT:
      case token_type::KEYWORD_IN:
      case token_type::KEYWORD_MACROS:
      case token_type::KEYWORD_NOT:
      case token_type::KEYWORD_OR:
      case token_type::KEYWORD_PASS:
      case token_type::KEYWORD_RETURN:
      case token_type::KEYWORD_RUNTIMEFEATURE:
      case token_type::KEYWORD_STRUCT:
      case token_type::KEYWORD_TRY:
      case token_type::KEYWORD_WHILE:
        is_kw = true;
      default:
        break;
    }
    return is_kw;
  }
  /**
 * Convert a token to a string
 * @param t token type
 * @return token type as a std::string
 */
  static inline std::string token_to_str(const token_type &t) {
    if (t == token_type::AND) return "AND";
    if (t == token_type::AND_EQ) return "AND_EQ";
    if (t == token_type::ARROW) return "ARROW";
    if (t == token_type::AT) return "AT";
    if (t == token_type::BA_DEDENT) return "BA_DEDENT";
    if (t == token_type::BA_INDENT) return "BA_INDENT";
    if (t == token_type::COLON) return "COLON";
    if (t == token_type::COMMA) return "COMMA";
    if (t == token_type::COMMENT) return "COMMENT";
    if (t == token_type::CURLY_BRACKET_CLOSE) return "CURLY_BRACKET_CLOSE";
    if (t == token_type::CURLY_BRACKET_OPEN) return "CURLY_BRACKET_OPEN";
    if (t == token_type::DIV) return "DIV";
    if (t == token_type::DIV_EQ) return "DIV_EQ";
    if (t == token_type::DOT) return "DOT";
    if (t == token_type::DOUBLE_NUMBER) return "DOUBLE_NUMBER";
    if (t == token_type::ELLIPSIS) return "ELLIPSIS";
    if (t == token_type::END_OF_FILE) return "END_OF_FILE";
    if (t == token_type::EQ) return "EQ";
    if (t == token_type::EQ_EQ) return "EQ_EQ";
    if (t == token_type::FLOAT_NUMBER) return "FLOAT_NUMBER";
    if (t == token_type::GREAT) return "GREAT";
    if (t == token_type::GREAT_EQ) return "GREAT_EQ";
    if (t == token_type::INDENT) return "INDENT";
    if (t == token_type::INTEGER_BIN) return "INTEGER_BIN";
    if (t == token_type::INTEGER_BIN_16) return "INTEGER_BIN_16";
    if (t == token_type::INTEGER_BIN_64) return "INTEGER_BIN_64";
    if (t == token_type::INTEGER_BIN_8) return "INTEGER_BIN_8";
    if (t == token_type::INTEGER_DECIMAL) return "INTEGER_DECIMAL";
    if (t == token_type::INTEGER_DECIMAL_16) return "INTEGER_DECIMAL_16";
    if (t == token_type::INTEGER_DECIMAL_64) return "INTEGER_DECIMAL_64";
    if (t == token_type::INTEGER_DECIMAL_8) return "INTEGER_DECIMAL_8";
    if (t == token_type::INTEGER_HEX) return "INTEGER_HEX";
    if (t == token_type::INTEGER_HEX_16) return "INTEGER_HEX_16";
    if (t == token_type::INTEGER_HEX_64) return "INTEGER_HEX_64";
    if (t == token_type::INTEGER_HEX_8) return "INTEGER_HEX_8";
    if (t == token_type::INTEGER_OCT) return "INTEGER_OCT";
    if (t == token_type::INTEGER_OCT_16) return "INTEGER_OCT_16";
    if (t == token_type::INTEGER_OCT_64) return "INTEGER_OCT_64";
    if (t == token_type::INTEGER_OCT_8) return "INTEGER_OCT_8";
    if (t == token_type::INT_DIV) return "INT_DIV";
    if (t == token_type::INT_DIV_EQ) return "INT_DIV_EQ";
    if (t == token_type::LESS) return "LESS";
    if (t == token_type::LESS_EQ) return "LESS_EQ";
    if (t == token_type::MOD) return "MOD";
    if (t == token_type::MOD_EQ) return "MOD_EQ";
    if (t == token_type::MUL) return "MUL";
    if (t == token_type::MUL_EQ) return "MUL_EQ";
    if (t == token_type::NAME) return "NAME";
    if (t == token_type::NEW_LINE) return "NEW_LINE";
    if (t == token_type::NOT_EQ) return "NOT_EQ";
    if (t == token_type::NOT_SYMBOL) return "NOT_SYMBOL";
    if (t == token_type::OR) return "OR";
    if (t == token_type::OR_EQ) return "OR_EQ";
    if (t == token_type::PAREN_CLOSE) return "PAREN_CLOSE";
    if (t == token_type::PAREN_OPEN) return "PAREN_OPEN";
    if (t == token_type::PLUS) return "PLUS";
    if (t == token_type::PLUS_EQ) return "PLUS_EQ";
    if (t == token_type::POWER) return "POWER";
    if (t == token_type::POWER_EQ) return "POWER_EQ";
    if (t == token_type::SEMICOLON) return "SEMICOLON";
    if (t == token_type::SHL) return "SHL";
    if (t == token_type::SHL_EQ) return "SHL_EQ";
    if (t == token_type::SHR) return "SHR";
    if (t == token_type::SHR_EQ) return "SHR_EQ";
    if (t == token_type::SQUARE_BRACKET_CLOSE) return "SQUARE_BRACKET_CLOSE";
    if (t == token_type::SQUARE_BRACKET_OPEN) return "SQUARE_BRACKET_OPEN";
    if (t == token_type::STRING) return "STRING";
    if (t == token_type::SUB) return "SUB";
    if (t == token_type::SUB_EQ) return "SUB_EQ";
    if (t == token_type::THREE_QUOTE_STRING) return "THREE_QUOTE_STRING";
    if (t == token_type::TILDE) return "TILDE";
    if (t == token_type::UINTEGER_BIN) return "UINTEGER_BIN";
    if (t == token_type::UINTEGER_BIN_16) return "UINTEGER_BIN_16";
    if (t == token_type::UINTEGER_BIN_64) return "UINTEGER_BIN_64";
    if (t == token_type::UINTEGER_BIN_8) return "UINTEGER_BIN_8";
    if (t == token_type::UINTEGER_DECIMAL) return "UINTEGER_DECIMAL";
    if (t == token_type::UINTEGER_DECIMAL_16) return "UINTEGER_DECIMAL_16";
    if (t == token_type::UINTEGER_DECIMAL_64) return "UINTEGER_DECIMAL_64";
    if (t == token_type::UINTEGER_DECIMAL_8) return "UINTEGER_DECIMAL_8";
    if (t == token_type::UINTEGER_HEX) return "UINTEGER_HEX";
    if (t == token_type::UINTEGER_HEX_16) return "UINTEGER_HEX_16";
    if (t == token_type::UINTEGER_HEX_64) return "UINTEGER_HEX_64";
    if (t == token_type::UINTEGER_HEX_8) return "UINTEGER_HEX_8";
    if (t == token_type::UINTEGER_OCT) return "UINTEGER_OCT";
    if (t == token_type::UINTEGER_OCT_16) return "UINTEGER_OCT_16";
    if (t == token_type::UINTEGER_OCT_64) return "UINTEGER_OCT_64";
    if (t == token_type::UINTEGER_OCT_8) return "UINTEGER_OCT_8";
    if (t == token_type::UNKNOWN_BIN) return "UNKNOWN_BIN";
    if (t == token_type::UNKNOWN_DECIMAL) return "UNKNOWN_DECIMAL";
    if (t == token_type::UNKNOWN_HEX) return "UNKNOWN_HEX";
    if (t == token_type::UNKNOWN_OCT) return "UNKNOWN_OCT";
    if (t == token_type::XOR) return "XOR";
    if (t == token_type::XOR_EQ) return "XOR_EQ";
    if (t == token_type::KEYWORD_FALSE) return "KEYWORD_FALSE";
    if (t == token_type::KEYWORD_NONE) return "KEYWORD_NONE";
    if (t == token_type::KEYWORD_TRUE) return "KEYWORD_TRUE";
    if (t == token_type::KEYWORD_AND) return "KEYWORD_AND";
    if (t == token_type::KEYWORD_AS) return "KEYWORD_AS";
    if (t == token_type::KEYWORD_ASSERT) return "KEYWORD_ASSERT";
    if (t == token_type::KEYWORD_BREAK) return "KEYWORD_BREAK";
    if (t == token_type::KEYWORD_CCODE) return "KEYWORD_CCODE";
    if (t == token_type::KEYWORD_CLASS) return "KEYWORD_CLASS";
    if (t == token_type::KEYWORD_CONTINUE) return "KEYWORD_CONTINUE";
    if (t == token_type::KEYWORD_DEF) return "KEYWORD_DEF";
    if (t == token_type::KEYWORD_DEFER) return "KEYWORD_DEFER";
    if (t == token_type::KEYWORD_DEL) return "KEYWORD_DEL";
    if (t == token_type::KEYWORD_DIRECTIVE) return "KEYWORD_DIRECTIVE";
    if (t == token_type::KEYWORD_ELIF) return "KEYWORD_ELIF";
    if (t == token_type::KEYWORD_ELSE) return "KEYWORD_ELSE";
    if (t == token_type::KEYWORD_FOR) return "KEYWORD_FOR";
    if (t == token_type::KEYWORD_FROM) return "KEYWORD_FROM";
    if (t == token_type::KEYWORD_IF) return "KEYWORD_IF";
    if (t == token_type::KEYWORD_IMPORT) return "KEYWORD_IMPORT";
    if (t == token_type::KEYWORD_IN) return "KEYWORD_IN";
    if (t == token_type::KEYWORD_MACROS) return "KEYWORD_MACROS";
    if (t == token_type::KEYWORD_NOT) return "KEYWORD_NOT";
    if (t == token_type::KEYWORD_OR) return "KEYWORD_OR";
    if (t == token_type::KEYWORD_PASS) return "KEYWORD_PASS";
    if (t == token_type::KEYWORD_RETURN) return "KEYWORD_RETURN";
    if (t == token_type::KEYWORD_RUNTIMEFEATURE)
      return "KEYWORD_RUNTIMEFEATURE";
    if (t == token_type::KEYWORD_STRUCT) return "KEYWORD_STRUCT";
    if (t == token_type::KEYWORD_TRY) return "KEYWORD_TRY";
    if (t == token_type::KEYWORD_WHILE) return "KEYWORD_WHILE";
    return "TK_UNKNOWN_TOKEN_DETECTED";
  }
  static inline token_type str_to_token(const std::string &t) {
    if (t == "AND") return token_type::AND;
    if (t == "AND_EQ") return token_type::AND_EQ;
    if (t == "ARROW") return token_type::ARROW;
    if (t == "AT") return token_type::AT;
    if (t == "BA_DEDENT") return token_type::BA_DEDENT;
    if (t == "BA_INDENT") return token_type::BA_INDENT;
    if (t == "COLON") return token_type::COLON;
    if (t == "COMMA") return token_type::COMMA;
    if (t == "COMMENT") return token_type::COMMENT;
    if (t == "CURLY_BRACKET_CLOSE") return token_type::CURLY_BRACKET_CLOSE;
    if (t == "CURLY_BRACKET_OPEN") return token_type::CURLY_BRACKET_OPEN;
    if (t == "DIV") return token_type::DIV;
    if (t == "DIV_EQ") return token_type::DIV_EQ;
    if (t == "DOT") return token_type::DOT;
    if (t == "DOUBLE_NUMBER") return token_type::DOUBLE_NUMBER;
    if (t == "ELLIPSIS") return token_type::ELLIPSIS;
    if (t == "END_OF_FILE") return token_type::END_OF_FILE;
    if (t == "EQ") return token_type::EQ;
    if (t == "EQ_EQ") return token_type::EQ_EQ;
    if (t == "FLOAT_NUMBER") return token_type::FLOAT_NUMBER;
    if (t == "GREAT") return token_type::GREAT;
    if (t == "GREAT_EQ") return token_type::GREAT_EQ;
    if (t == "INDENT") return token_type::INDENT;
    if (t == "INTEGER_BIN") return token_type::INTEGER_BIN;
    if (t == "INTEGER_BIN_16") return token_type::INTEGER_BIN_16;
    if (t == "INTEGER_BIN_64") return token_type::INTEGER_BIN_64;
    if (t == "INTEGER_BIN_8") return token_type::INTEGER_BIN_8;
    if (t == "INTEGER_DECIMAL") return token_type::INTEGER_DECIMAL;
    if (t == "INTEGER_DECIMAL_16") return token_type::INTEGER_DECIMAL_16;
    if (t == "INTEGER_DECIMAL_64") return token_type::INTEGER_DECIMAL_64;
    if (t == "INTEGER_DECIMAL_8") return token_type::INTEGER_DECIMAL_8;
    if (t == "INTEGER_HEX") return token_type::INTEGER_HEX;
    if (t == "INTEGER_HEX_16") return token_type::INTEGER_HEX_16;
    if (t == "INTEGER_HEX_64") return token_type::INTEGER_HEX_64;
    if (t == "INTEGER_HEX_8") return token_type::INTEGER_HEX_8;
    if (t == "INTEGER_OCT") return token_type::INTEGER_OCT;
    if (t == "INTEGER_OCT_16") return token_type::INTEGER_OCT_16;
    if (t == "INTEGER_OCT_64") return token_type::INTEGER_OCT_64;
    if (t == "INTEGER_OCT_8") return token_type::INTEGER_OCT_8;
    if (t == "INT_DIV") return token_type::INT_DIV;
    if (t == "INT_DIV_EQ") return token_type::INT_DIV_EQ;
    if (t == "LESS") return token_type::LESS;
    if (t == "LESS_EQ") return token_type::LESS_EQ;
    if (t == "MOD") return token_type::MOD;
    if (t == "MOD_EQ") return token_type::MOD_EQ;
    if (t == "MUL") return token_type::MUL;
    if (t == "MUL_EQ") return token_type::MUL_EQ;
    if (t == "NAME") return token_type::NAME;
    if (t == "NEW_LINE") return token_type::NEW_LINE;
    if (t == "NOT_EQ") return token_type::NOT_EQ;
    if (t == "NOT_SYMBOL") return token_type::NOT_SYMBOL;
    if (t == "OR") return token_type::OR;
    if (t == "OR_EQ") return token_type::OR_EQ;
    if (t == "PAREN_CLOSE") return token_type::PAREN_CLOSE;
    if (t == "PAREN_OPEN") return token_type::PAREN_OPEN;
    if (t == "PLUS") return token_type::PLUS;
    if (t == "PLUS_EQ") return token_type::PLUS_EQ;
    if (t == "POWER") return token_type::POWER;
    if (t == "POWER_EQ") return token_type::POWER_EQ;
    if (t == "SEMICOLON") return token_type::SEMICOLON;
    if (t == "SHL") return token_type::SHL;
    if (t == "SHL_EQ") return token_type::SHL_EQ;
    if (t == "SHR") return token_type::SHR;
    if (t == "SHR_EQ") return token_type::SHR_EQ;
    if (t == "SQUARE_BRACKET_CLOSE") return token_type::SQUARE_BRACKET_CLOSE;
    if (t == "SQUARE_BRACKET_OPEN") return token_type::SQUARE_BRACKET_OPEN;
    if (t == "STRING") return token_type::STRING;
    if (t == "SUB") return token_type::SUB;
    if (t == "SUB_EQ") return token_type::SUB_EQ;
    if (t == "THREE_QUOTE_STRING") return token_type::THREE_QUOTE_STRING;
    if (t == "TILDE") return token_type::TILDE;
    if (t == "UINTEGER_BIN") return token_type::UINTEGER_BIN;
    if (t == "UINTEGER_BIN_16") return token_type::UINTEGER_BIN_16;
    if (t == "UINTEGER_BIN_64") return token_type::UINTEGER_BIN_64;
    if (t == "UINTEGER_BIN_8") return token_type::UINTEGER_BIN_8;
    if (t == "UINTEGER_DECIMAL") return token_type::UINTEGER_DECIMAL;
    if (t == "UINTEGER_DECIMAL_16") return token_type::UINTEGER_DECIMAL_16;
    if (t == "UINTEGER_DECIMAL_64") return token_type::UINTEGER_DECIMAL_64;
    if (t == "UINTEGER_DECIMAL_8") return token_type::UINTEGER_DECIMAL_8;
    if (t == "UINTEGER_HEX") return token_type::UINTEGER_HEX;
    if (t == "UINTEGER_HEX_16") return token_type::UINTEGER_HEX_16;
    if (t == "UINTEGER_HEX_64") return token_type::UINTEGER_HEX_64;
    if (t == "UINTEGER_HEX_8") return token_type::UINTEGER_HEX_8;
    if (t == "UINTEGER_OCT") return token_type::UINTEGER_OCT;
    if (t == "UINTEGER_OCT_16") return token_type::UINTEGER_OCT_16;
    if (t == "UINTEGER_OCT_64") return token_type::UINTEGER_OCT_64;
    if (t == "UINTEGER_OCT_8") return token_type::UINTEGER_OCT_8;
    if (t == "UNKNOWN_BIN") return token_type::UNKNOWN_BIN;
    if (t == "UNKNOWN_DECIMAL") return token_type::UNKNOWN_DECIMAL;
    if (t == "UNKNOWN_HEX") return token_type::UNKNOWN_HEX;
    if (t == "UNKNOWN_OCT") return token_type::UNKNOWN_OCT;
    if (t == "XOR") return token_type::XOR;
    if (t == "XOR_EQ") return token_type::XOR_EQ;
    if (t == "KEYWORD_FALSE") return token_type::KEYWORD_FALSE;
    if (t == "KEYWORD_NONE") return token_type::KEYWORD_NONE;
    if (t == "KEYWORD_TRUE") return token_type::KEYWORD_TRUE;
    if (t == "KEYWORD_AND") return token_type::KEYWORD_AND;
    if (t == "KEYWORD_AS") return token_type::KEYWORD_AS;
    if (t == "KEYWORD_ASSERT") return token_type::KEYWORD_ASSERT;
    if (t == "KEYWORD_BREAK") return token_type::KEYWORD_BREAK;
    if (t == "KEYWORD_CCODE") return token_type::KEYWORD_CCODE;
    if (t == "KEYWORD_CLASS") return token_type::KEYWORD_CLASS;
    if (t == "KEYWORD_CONTINUE") return token_type::KEYWORD_CONTINUE;
    if (t == "KEYWORD_DEF") return token_type::KEYWORD_DEF;
    if (t == "KEYWORD_DEFER") return token_type::KEYWORD_DEFER;
    if (t == "KEYWORD_DEL") return token_type::KEYWORD_DEL;
    if (t == "KEYWORD_DIRECTIVE") return token_type::KEYWORD_DIRECTIVE;
    if (t == "KEYWORD_ELIF") return token_type::KEYWORD_ELIF;
    if (t == "KEYWORD_ELSE") return token_type::KEYWORD_ELSE;
    if (t == "KEYWORD_FOR") return token_type::KEYWORD_FOR;
    if (t == "KEYWORD_FROM") return token_type::KEYWORD_FROM;
    if (t == "KEYWORD_IF") return token_type::KEYWORD_IF;
    if (t == "KEYWORD_IMPORT") return token_type::KEYWORD_IMPORT;
    if (t == "KEYWORD_IN") return token_type::KEYWORD_IN;
    if (t == "KEYWORD_MACROS") return token_type::KEYWORD_MACROS;
    if (t == "KEYWORD_NOT") return token_type::KEYWORD_NOT;
    if (t == "KEYWORD_OR") return token_type::KEYWORD_OR;
    if (t == "KEYWORD_PASS") return token_type::KEYWORD_PASS;
    if (t == "KEYWORD_RETURN") return token_type::KEYWORD_RETURN;
    if (t == "KEYWORD_RUNTIMEFEATURE")
      return token_type::KEYWORD_RUNTIMEFEATURE;
    if (t == "KEYWORD_STRUCT") return token_type::KEYWORD_STRUCT;
    if (t == "KEYWORD_TRY") return token_type::KEYWORD_TRY;
    if (t == "KEYWORD_WHILE") return token_type::KEYWORD_WHILE;
    return token_type::TK_UNKNOWN_TOKEN_DETECTED;
  }
  static inline token_type str_to_keyword(const std::string &t) {
    if (t == "False") return token_type::KEYWORD_FALSE;
    if (t == "None") return token_type::KEYWORD_NONE;
    if (t == "True") return token_type::KEYWORD_TRUE;
    if (t == "and") return token_type::KEYWORD_AND;
    if (t == "as") return token_type::KEYWORD_AS;
    if (t == "assert") return token_type::KEYWORD_ASSERT;
    if (t == "break") return token_type::KEYWORD_BREAK;
    if (t == "ccode") return token_type::KEYWORD_CCODE;
    if (t == "class") return token_type::KEYWORD_CLASS;
    if (t == "continue") return token_type::KEYWORD_CONTINUE;
    if (t == "def") return token_type::KEYWORD_DEF;
    if (t == "defer") return token_type::KEYWORD_DEFER;
    if (t == "del") return token_type::KEYWORD_DEL;
    if (t == "directive") return token_type::KEYWORD_DIRECTIVE;
    if (t == "elif") return token_type::KEYWORD_ELIF;
    if (t == "else") return token_type::KEYWORD_ELSE;
    if (t == "for") return token_type::KEYWORD_FOR;
    if (t == "from") return token_type::KEYWORD_FROM;
    if (t == "if") return token_type::KEYWORD_IF;
    if (t == "import") return token_type::KEYWORD_IMPORT;
    if (t == "in") return token_type::KEYWORD_IN;
    if (t == "macros") return token_type::KEYWORD_MACROS;
    if (t == "not") return token_type::KEYWORD_NOT;
    if (t == "or") return token_type::KEYWORD_OR;
    if (t == "pass") return token_type::KEYWORD_PASS;
    if (t == "return") return token_type::KEYWORD_RETURN;
    if (t == "runtimefeature") return token_type::KEYWORD_RUNTIMEFEATURE;
    if (t == "struct") return token_type::KEYWORD_STRUCT;
    if (t == "try") return token_type::KEYWORD_TRY;
    if (t == "while") return token_type::KEYWORD_WHILE;
    return token_type::TK_UNKNOWN_TOKEN_DETECTED;
  }
  static inline token_type numeric_id_to_token(const int64_t n) {
    if (n == 28864) return token_type::AND;
    if (n == 28865) return token_type::AND_EQ;
    if (n == 28866) return token_type::ARROW;
    if (n == 28867) return token_type::AT;
    if (n == 28868) return token_type::BA_DEDENT;
    if (n == 28869) return token_type::BA_INDENT;
    if (n == 28870) return token_type::COLON;
    if (n == 28871) return token_type::COMMA;
    if (n == 28873) return token_type::CURLY_BRACKET_CLOSE;
    if (n == 28874) return token_type::CURLY_BRACKET_OPEN;
    if (n == 28875) return token_type::DIV;
    if (n == 28876) return token_type::DIV_EQ;
    if (n == 28877) return token_type::DOT;
    if (n == 28878) return token_type::DOUBLE_NUMBER;
    if (n == 28879) return token_type::ELLIPSIS;
    if (n == 28881) return token_type::EQ;
    if (n == 28882) return token_type::EQ_EQ;
    if (n == 28883) return token_type::FLOAT_NUMBER;
    if (n == 28884) return token_type::GREAT;
    if (n == 28885) return token_type::GREAT_EQ;
    if (n == 28887) return token_type::INTEGER_BIN;
    if (n == 28888) return token_type::INTEGER_BIN_16;
    if (n == 28889) return token_type::INTEGER_BIN_64;
    if (n == 28890) return token_type::INTEGER_BIN_8;
    if (n == 28891) return token_type::INTEGER_DECIMAL;
    if (n == 28892) return token_type::INTEGER_DECIMAL_16;
    if (n == 28893) return token_type::INTEGER_DECIMAL_64;
    if (n == 28894) return token_type::INTEGER_DECIMAL_8;
    if (n == 28895) return token_type::INTEGER_HEX;
    if (n == 28896) return token_type::INTEGER_HEX_16;
    if (n == 28897) return token_type::INTEGER_HEX_64;
    if (n == 28898) return token_type::INTEGER_HEX_8;
    if (n == 28899) return token_type::INTEGER_OCT;
    if (n == 28900) return token_type::INTEGER_OCT_16;
    if (n == 28901) return token_type::INTEGER_OCT_64;
    if (n == 28902) return token_type::INTEGER_OCT_8;
    if (n == 28903) return token_type::INT_DIV;
    if (n == 28904) return token_type::INT_DIV_EQ;
    if (n == 28905) return token_type::LESS;
    if (n == 28906) return token_type::LESS_EQ;
    if (n == 28907) return token_type::MOD;
    if (n == 28908) return token_type::MOD_EQ;
    if (n == 28909) return token_type::MUL;
    if (n == 28910) return token_type::MUL_EQ;
    if (n == 28911) return token_type::NAME;
    if (n == 28912) return token_type::NEW_LINE;
    if (n == 28913) return token_type::NOT_EQ;
    if (n == 28914) return token_type::NOT_SYMBOL;
    if (n == 28915) return token_type::OR;
    if (n == 28916) return token_type::OR_EQ;
    if (n == 28917) return token_type::PAREN_CLOSE;
    if (n == 28918) return token_type::PAREN_OPEN;
    if (n == 28919) return token_type::PLUS;
    if (n == 28920) return token_type::PLUS_EQ;
    if (n == 28921) return token_type::POWER;
    if (n == 28922) return token_type::POWER_EQ;
    if (n == 28923) return token_type::SEMICOLON;
    if (n == 28924) return token_type::SHL;
    if (n == 28925) return token_type::SHL_EQ;
    if (n == 28926) return token_type::SHR;
    if (n == 28927) return token_type::SHR_EQ;
    if (n == 28928) return token_type::SQUARE_BRACKET_CLOSE;
    if (n == 28929) return token_type::SQUARE_BRACKET_OPEN;
    if (n == 28930) return token_type::STRING;
    if (n == 28931) return token_type::SUB;
    if (n == 28932) return token_type::SUB_EQ;
    if (n == 28933) return token_type::THREE_QUOTE_STRING;
    if (n == 28934) return token_type::TILDE;
    if (n == 28935) return token_type::UINTEGER_BIN;
    if (n == 28936) return token_type::UINTEGER_BIN_16;
    if (n == 28937) return token_type::UINTEGER_BIN_64;
    if (n == 28938) return token_type::UINTEGER_BIN_8;
    if (n == 28939) return token_type::UINTEGER_DECIMAL;
    if (n == 28940) return token_type::UINTEGER_DECIMAL_16;
    if (n == 28941) return token_type::UINTEGER_DECIMAL_64;
    if (n == 28942) return token_type::UINTEGER_DECIMAL_8;
    if (n == 28943) return token_type::UINTEGER_HEX;
    if (n == 28944) return token_type::UINTEGER_HEX_16;
    if (n == 28945) return token_type::UINTEGER_HEX_64;
    if (n == 28946) return token_type::UINTEGER_HEX_8;
    if (n == 28947) return token_type::UINTEGER_OCT;
    if (n == 28948) return token_type::UINTEGER_OCT_16;
    if (n == 28949) return token_type::UINTEGER_OCT_64;
    if (n == 28950) return token_type::UINTEGER_OCT_8;
    if (n == 28955) return token_type::XOR;
    if (n == 28956) return token_type::XOR_EQ;
    if (n == 28957) return token_type::KEYWORD_FALSE;
    if (n == 28958) return token_type::KEYWORD_NONE;
    if (n == 28959) return token_type::KEYWORD_TRUE;
    if (n == 28960) return token_type::KEYWORD_AND;
    if (n == 28961) return token_type::KEYWORD_AS;
    if (n == 28962) return token_type::KEYWORD_ASSERT;
    if (n == 28963) return token_type::KEYWORD_BREAK;
    if (n == 28964) return token_type::KEYWORD_CCODE;
    if (n == 28965) return token_type::KEYWORD_CLASS;
    if (n == 28966) return token_type::KEYWORD_CONTINUE;
    if (n == 28967) return token_type::KEYWORD_DEF;
    if (n == 28968) return token_type::KEYWORD_DEFER;
    if (n == 28969) return token_type::KEYWORD_DEL;
    if (n == 28970) return token_type::KEYWORD_DIRECTIVE;
    if (n == 28971) return token_type::KEYWORD_ELIF;
    if (n == 28972) return token_type::KEYWORD_ELSE;
    if (n == 28973) return token_type::KEYWORD_FOR;
    if (n == 28974) return token_type::KEYWORD_FROM;
    if (n == 28975) return token_type::KEYWORD_IF;
    if (n == 28976) return token_type::KEYWORD_IMPORT;
    if (n == 28977) return token_type::KEYWORD_IN;
    if (n == 28978) return token_type::KEYWORD_MACROS;
    if (n == 28979) return token_type::KEYWORD_NOT;
    if (n == 28980) return token_type::KEYWORD_OR;
    if (n == 28981) return token_type::KEYWORD_PASS;
    if (n == 28982) return token_type::KEYWORD_RETURN;
    if (n == 28983) return token_type::KEYWORD_RUNTIMEFEATURE;
    if (n == 28984) return token_type::KEYWORD_STRUCT;
    if (n == 28985) return token_type::KEYWORD_TRY;
    if (n == 28986) return token_type::KEYWORD_WHILE;
    return token_type::TK_UNKNOWN_TOKEN_DETECTED;
  }
  static inline int64_t token_to_numeric_id(const token_type &t) {
    if (t == token_type::AND) return 28864;
    if (t == token_type::AND_EQ) return 28865;
    if (t == token_type::ARROW) return 28866;
    if (t == token_type::AT) return 28867;
    if (t == token_type::BA_DEDENT) return 28868;
    if (t == token_type::BA_INDENT) return 28869;
    if (t == token_type::COLON) return 28870;
    if (t == token_type::COMMA) return 28871;
    if (t == token_type::CURLY_BRACKET_CLOSE) return 28873;
    if (t == token_type::CURLY_BRACKET_OPEN) return 28874;
    if (t == token_type::DIV) return 28875;
    if (t == token_type::DIV_EQ) return 28876;
    if (t == token_type::DOT) return 28877;
    if (t == token_type::DOUBLE_NUMBER) return 28878;
    if (t == token_type::ELLIPSIS) return 28879;
    if (t == token_type::EQ) return 28881;
    if (t == token_type::EQ_EQ) return 28882;
    if (t == token_type::FLOAT_NUMBER) return 28883;
    if (t == token_type::GREAT) return 28884;
    if (t == token_type::GREAT_EQ) return 28885;
    if (t == token_type::INTEGER_BIN) return 28887;
    if (t == token_type::INTEGER_BIN_16) return 28888;
    if (t == token_type::INTEGER_BIN_64) return 28889;
    if (t == token_type::INTEGER_BIN_8) return 28890;
    if (t == token_type::INTEGER_DECIMAL) return 28891;
    if (t == token_type::INTEGER_DECIMAL_16) return 28892;
    if (t == token_type::INTEGER_DECIMAL_64) return 28893;
    if (t == token_type::INTEGER_DECIMAL_8) return 28894;
    if (t == token_type::INTEGER_HEX) return 28895;
    if (t == token_type::INTEGER_HEX_16) return 28896;
    if (t == token_type::INTEGER_HEX_64) return 28897;
    if (t == token_type::INTEGER_HEX_8) return 28898;
    if (t == token_type::INTEGER_OCT) return 28899;
    if (t == token_type::INTEGER_OCT_16) return 28900;
    if (t == token_type::INTEGER_OCT_64) return 28901;
    if (t == token_type::INTEGER_OCT_8) return 28902;
    if (t == token_type::INT_DIV) return 28903;
    if (t == token_type::INT_DIV_EQ) return 28904;
    if (t == token_type::LESS) return 28905;
    if (t == token_type::LESS_EQ) return 28906;
    if (t == token_type::MOD) return 28907;
    if (t == token_type::MOD_EQ) return 28908;
    if (t == token_type::MUL) return 28909;
    if (t == token_type::MUL_EQ) return 28910;
    if (t == token_type::NAME) return 28911;
    if (t == token_type::NEW_LINE) return 28912;
    if (t == token_type::NOT_EQ) return 28913;
    if (t == token_type::NOT_SYMBOL) return 28914;
    if (t == token_type::OR) return 28915;
    if (t == token_type::OR_EQ) return 28916;
    if (t == token_type::PAREN_CLOSE) return 28917;
    if (t == token_type::PAREN_OPEN) return 28918;
    if (t == token_type::PLUS) return 28919;
    if (t == token_type::PLUS_EQ) return 28920;
    if (t == token_type::POWER) return 28921;
    if (t == token_type::POWER_EQ) return 28922;
    if (t == token_type::SEMICOLON) return 28923;
    if (t == token_type::SHL) return 28924;
    if (t == token_type::SHL_EQ) return 28925;
    if (t == token_type::SHR) return 28926;
    if (t == token_type::SHR_EQ) return 28927;
    if (t == token_type::SQUARE_BRACKET_CLOSE) return 28928;
    if (t == token_type::SQUARE_BRACKET_OPEN) return 28929;
    if (t == token_type::STRING) return 28930;
    if (t == token_type::SUB) return 28931;
    if (t == token_type::SUB_EQ) return 28932;
    if (t == token_type::THREE_QUOTE_STRING) return 28933;
    if (t == token_type::TILDE) return 28934;
    if (t == token_type::UINTEGER_BIN) return 28935;
    if (t == token_type::UINTEGER_BIN_16) return 28936;
    if (t == token_type::UINTEGER_BIN_64) return 28937;
    if (t == token_type::UINTEGER_BIN_8) return 28938;
    if (t == token_type::UINTEGER_DECIMAL) return 28939;
    if (t == token_type::UINTEGER_DECIMAL_16) return 28940;
    if (t == token_type::UINTEGER_DECIMAL_64) return 28941;
    if (t == token_type::UINTEGER_DECIMAL_8) return 28942;
    if (t == token_type::UINTEGER_HEX) return 28943;
    if (t == token_type::UINTEGER_HEX_16) return 28944;
    if (t == token_type::UINTEGER_HEX_64) return 28945;
    if (t == token_type::UINTEGER_HEX_8) return 28946;
    if (t == token_type::UINTEGER_OCT) return 28947;
    if (t == token_type::UINTEGER_OCT_16) return 28948;
    if (t == token_type::UINTEGER_OCT_64) return 28949;
    if (t == token_type::UINTEGER_OCT_8) return 28950;
    if (t == token_type::XOR) return 28955;
    if (t == token_type::XOR_EQ) return 28956;
    if (t == token_type::KEYWORD_FALSE) return 28957;
    if (t == token_type::KEYWORD_NONE) return 28958;
    if (t == token_type::KEYWORD_TRUE) return 28959;
    if (t == token_type::KEYWORD_AND) return 28960;
    if (t == token_type::KEYWORD_AS) return 28961;
    if (t == token_type::KEYWORD_ASSERT) return 28962;
    if (t == token_type::KEYWORD_BREAK) return 28963;
    if (t == token_type::KEYWORD_CCODE) return 28964;
    if (t == token_type::KEYWORD_CLASS) return 28965;
    if (t == token_type::KEYWORD_CONTINUE) return 28966;
    if (t == token_type::KEYWORD_DEF) return 28967;
    if (t == token_type::KEYWORD_DEFER) return 28968;
    if (t == token_type::KEYWORD_DEL) return 28969;
    if (t == token_type::KEYWORD_DIRECTIVE) return 28970;
    if (t == token_type::KEYWORD_ELIF) return 28971;
    if (t == token_type::KEYWORD_ELSE) return 28972;
    if (t == token_type::KEYWORD_FOR) return 28973;
    if (t == token_type::KEYWORD_FROM) return 28974;
    if (t == token_type::KEYWORD_IF) return 28975;
    if (t == token_type::KEYWORD_IMPORT) return 28976;
    if (t == token_type::KEYWORD_IN) return 28977;
    if (t == token_type::KEYWORD_MACROS) return 28978;
    if (t == token_type::KEYWORD_NOT) return 28979;
    if (t == token_type::KEYWORD_OR) return 28980;
    if (t == token_type::KEYWORD_PASS) return 28981;
    if (t == token_type::KEYWORD_RETURN) return 28982;
    if (t == token_type::KEYWORD_RUNTIMEFEATURE) return 28983;
    if (t == token_type::KEYWORD_STRUCT) return 28984;
    if (t == token_type::KEYWORD_TRY) return 28985;
    if (t == token_type::KEYWORD_WHILE) return 28986;
    return 0;
  }
  static inline bool is_number_token(const token_type &t) {
    return (
        t == token_type::DOUBLE_NUMBER || t == token_type::FLOAT_NUMBER ||
        t == token_type::INTEGER_BIN || t == token_type::INTEGER_BIN_16 ||
        t == token_type::INTEGER_BIN_64 || t == token_type::INTEGER_BIN_8 ||
        t == token_type::INTEGER_DECIMAL ||
        t == token_type::INTEGER_DECIMAL_16 ||
        t == token_type::INTEGER_DECIMAL_64 ||
        t == token_type::INTEGER_DECIMAL_8 || t == token_type::INTEGER_HEX ||
        t == token_type::INTEGER_HEX_16 || t == token_type::INTEGER_HEX_64 ||
        t == token_type::INTEGER_HEX_8 || t == token_type::INTEGER_OCT ||
        t == token_type::INTEGER_OCT_16 || t == token_type::INTEGER_OCT_64 ||
        t == token_type::INTEGER_OCT_8 || t == token_type::UINTEGER_BIN ||
        t == token_type::UINTEGER_BIN_16 || t == token_type::UINTEGER_BIN_64 ||
        t == token_type::UINTEGER_BIN_8 || t == token_type::UINTEGER_DECIMAL ||
        t == token_type::UINTEGER_DECIMAL_16 ||
        t == token_type::UINTEGER_DECIMAL_64 ||
        t == token_type::UINTEGER_DECIMAL_8 || t == token_type::UINTEGER_HEX ||
        t == token_type::UINTEGER_HEX_16 || t == token_type::UINTEGER_HEX_64 ||
        t == token_type::UINTEGER_HEX_8 || t == token_type::UINTEGER_OCT ||
        t == token_type::UINTEGER_OCT_16 || t == token_type::UINTEGER_OCT_64 ||
        t == token_type::UINTEGER_OCT_8);
  }
  static inline bool is_valid_complex(const std::string &value,
                                      const token_type &t) {
    if (t == token_type::STRING || t == token_type::THREE_QUOTE_STRING ||
        t == token_type::COMMENT || t == token_type::NEW_LINE ||
        t == token_type::END_OF_FILE || t == token_type::BA_INDENT ||
        t == token_type::BA_DEDENT || t == token_type::INDENT) {
      return true;
    }
    if (t == token_type::DOUBLE_NUMBER) {
      try {
        std::stod(value);
        return true;
      } catch (...) { return false; }
    }
    if (t == token_type::FLOAT_NUMBER) {
      if (value.size() > 2 &&
          (value[value.size() - 1] == 'f' || value[value.size() - 1] == 'F')) {
        for (size_t i = 0; i < value.size() - 1; i++) {
          if (value[i] < '0' || value[i] > '9') {
            std::string new_value = value.substr(0, value.size() - 1);
            return is_valid_complex(new_value, token_type::DOUBLE_NUMBER);
          }
        }
        return true;// This is a short float - 1f, 2f, etc
      }
      return false;
    }
    if (t == token_type::INTEGER_BIN || t == token_type::INTEGER_BIN_8 ||
        t == token_type::INTEGER_BIN_16 || t == token_type::INTEGER_BIN_64 ||
        t == token_type::UINTEGER_BIN || t == token_type::UINTEGER_BIN_8 ||
        t == token_type::UINTEGER_BIN_16 || t == token_type::UINTEGER_BIN_64) {
      if (value.size() < 3) return false;
      if (value[0] != '0' || (value[1] != 'b' && value[1] != 'B')) return false;
      for (size_t i = 2; i < value.size(); i++) {
        if (value[i] != '0' && value[i] != '1') return false;
      }
      return true;
    }
    if (t == token_type::INTEGER_DECIMAL ||
        t == token_type::INTEGER_DECIMAL_8 ||
        t == token_type::INTEGER_DECIMAL_16 ||
        t == token_type::INTEGER_DECIMAL_64 ||
        t == token_type::UINTEGER_DECIMAL ||
        t == token_type::UINTEGER_DECIMAL_8 ||
        t == token_type::UINTEGER_DECIMAL_16 ||
        t == token_type::UINTEGER_DECIMAL_64) {
      if (value.empty()) return false;
      if (value == "0") return true;
      if (value[0] < '1' || value[0] > '9') return false;
      for (size_t i = 1; i < value.size(); i++) {
        if (value[i] < '0' || value[i] > '9') return false;
      }
      return true;
    }
    if (t == token_type::INTEGER_HEX || t == token_type::INTEGER_HEX_8 ||
        t == token_type::INTEGER_HEX_16 || t == token_type::INTEGER_HEX_64 ||
        t == token_type::UINTEGER_HEX || t == token_type::UINTEGER_HEX_8 ||
        t == token_type::UINTEGER_HEX_16 || t == token_type::UINTEGER_HEX_64) {
      if (value.size() < 3) return false;
      if (value[0] != '0' || (value[1] != 'x' && value[1] != 'X')) return false;
      for (size_t i = 2; i < value.size(); i++) {
        if ((value[i] < '0' || value[i] > '9') &&
            (value[i] < 'a' || value[i] > 'f') &&
            (value[i] < 'A' || value[i] > 'F'))
          return false;
      }
      return true;
    }
    if (t == token_type::INTEGER_OCT || t == token_type::INTEGER_OCT_8 ||
        t == token_type::INTEGER_OCT_16 || t == token_type::INTEGER_OCT_64 ||
        t == token_type::UINTEGER_OCT || t == token_type::UINTEGER_OCT_8 ||
        t == token_type::UINTEGER_OCT_16 || t == token_type::UINTEGER_OCT_64) {
      if (value.size() < 3) return false;
      if (value[0] != '0' || (value[1] != 'o' && value[1] != 'O')) return false;
      for (size_t i = 2; i < value.size(); i++) {
        if (value[i] < '0' || value[i] > '7') return false;
      }
      return true;
    }
    if (t == token_type::NAME) {
      if (value.empty()) return false;
      std::string new_value = value;
      // Can start with a $ for gensym
      if (value[0] == '$') new_value = value.substr(1);
      if (new_value.empty()) return false;
      if (new_value[0] != '_' && (new_value[0] < 'a' || new_value[0] > 'z') &&
          (new_value[0] < 'A' || new_value[0] > 'Z'))
        return false;
      for (size_t i = 1; i < new_value.size(); i++) {
        if (new_value[i] != '_' && (new_value[i] < 'a' || new_value[i] > 'z') &&
            (new_value[i] < 'A' || new_value[i] > 'Z') &&
            (new_value[i] < '0' || new_value[i] > '9'))
          return false;
      }
      return true;
    }
    return false;
  }
  static inline bool is_valid(const std::string &value, const token_type &t) {
    if (t == token_type::AT) return (value == "@");
    if (t == token_type::COLON) return (value == ":");
    if (t == token_type::SEMICOLON) return (value == ";");
    if (t == token_type::PAREN_OPEN) return (value == "(");
    if (t == token_type::PAREN_CLOSE) return (value == ")");
    if (t == token_type::SQUARE_BRACKET_OPEN) return (value == "[");
    if (t == token_type::SQUARE_BRACKET_CLOSE) return (value == "]");
    if (t == token_type::CURLY_BRACKET_OPEN) return (value == "{");
    if (t == token_type::CURLY_BRACKET_CLOSE) return (value == "}");
    if (t == token_type::LESS) return (value == "<");
    if (t == token_type::EQ) return (value == "=");
    if (t == token_type::GREAT) return (value == ">");
    if (t == token_type::OR) return (value == "|");
    if (t == token_type::PLUS) return (value == "+");
    if (t == token_type::SUB) return (value == "-");
    if (t == token_type::MUL) return (value == "*");
    if (t == token_type::DIV) return (value == "/");
    if (t == token_type::AND) return (value == "&");
    if (t == token_type::XOR) return (value == "^");
    if (t == token_type::MOD) return (value == "%");
    if (t == token_type::INT_DIV) return (value == "//");
    if (t == token_type::POWER) return (value == "**");
    if (t == token_type::LESS_EQ) return (value == "<=");
    if (t == token_type::EQ_EQ) return (value == "==");
    if (t == token_type::GREAT_EQ) return (value == ">=");
    if (t == token_type::OR_EQ) return (value == "|=");
    if (t == token_type::PLUS_EQ) return (value == "+=");
    if (t == token_type::SUB_EQ) return (value == "-=");
    if (t == token_type::MUL_EQ) return (value == "*=");
    if (t == token_type::DIV_EQ) return (value == "/=");
    if (t == token_type::AND_EQ) return (value == "&=");
    if (t == token_type::XOR_EQ) return (value == "^=");
    if (t == token_type::MOD_EQ) return (value == "%=");
    if (t == token_type::INT_DIV_EQ) return (value == "//=");
    if (t == token_type::POWER_EQ) return (value == "**=");
    if (t == token_type::ARROW) return (value == "->");
    if (t == token_type::ELLIPSIS) return (value == "...");
    if (t == token_type::DOT) return (value == ".");
    if (t == token_type::COMMA) return (value == ",");
    if (t == token_type::SHL) return (value == "<<");
    if (t == token_type::SHR) return (value == ">>");
    if (t == token_type::SHL_EQ) return (value == "<<=");
    if (t == token_type::SHR_EQ) return (value == ">>=");
    if (t == token_type::NOT_EQ) return (value == "!=");
    if (t == token_type::TILDE) return (value == "~");
    if (t == token_type::NOT_SYMBOL) return (value == "!");
    if (t == token_type::KEYWORD_FALSE) return (value == "False");
    if (t == token_type::KEYWORD_NONE) return (value == "None");
    if (t == token_type::KEYWORD_TRUE) return (value == "True");
    if (t == token_type::KEYWORD_AND) return (value == "and");
    if (t == token_type::KEYWORD_AS) return (value == "as");
    if (t == token_type::KEYWORD_ASSERT) return (value == "assert");
    if (t == token_type::KEYWORD_BREAK) return (value == "break");
    if (t == token_type::KEYWORD_CCODE) return (value == "ccode");
    if (t == token_type::KEYWORD_CLASS) return (value == "class");
    if (t == token_type::KEYWORD_CONTINUE) return (value == "continue");
    if (t == token_type::KEYWORD_DEF) return (value == "def");
    if (t == token_type::KEYWORD_DEFER) return (value == "defer");
    if (t == token_type::KEYWORD_DEL) return (value == "del");
    if (t == token_type::KEYWORD_DIRECTIVE) return (value == "directive");
    if (t == token_type::KEYWORD_ELIF) return (value == "elif");
    if (t == token_type::KEYWORD_ELSE) return (value == "else");
    if (t == token_type::KEYWORD_FOR) return (value == "for");
    if (t == token_type::KEYWORD_FROM) return (value == "from");
    if (t == token_type::KEYWORD_IF) return (value == "if");
    if (t == token_type::KEYWORD_IMPORT) return (value == "import");
    if (t == token_type::KEYWORD_IN) return (value == "in");
    if (t == token_type::KEYWORD_MACROS) return (value == "macros");
    if (t == token_type::KEYWORD_NOT) return (value == "not");
    if (t == token_type::KEYWORD_OR) return (value == "or");
    if (t == token_type::KEYWORD_PASS) return (value == "pass");
    if (t == token_type::KEYWORD_RETURN) return (value == "return");
    if (t == token_type::KEYWORD_RUNTIMEFEATURE)
      return (value == "runtimefeature");
    if (t == token_type::KEYWORD_STRUCT) return (value == "struct");
    if (t == token_type::KEYWORD_TRY) return (value == "try");
    if (t == token_type::KEYWORD_WHILE) return (value == "while");
    return is_valid_complex(value, t);
  }
  /**
 * Single token
 */
  struct token {
    std::string file_;// filename
    int line_;        // line in file
    int pos_;         // position in line
    std::string token_;
    token_type type_;// type of the token
    std::string original_;
    // For gc_pool
    token *next_{nullptr};
    token *prev_{nullptr};
    std::uint8_t mark_{0};
    void clean_state();
  };
  /**
 * Content of an error message
 */
  struct parsing_error : public std::exception {
    std::string message_;// content of the line this error occurred
    token tok_;
    bool token_set_{};
    parsing_error(std::string message, token *at);
    parsing_error(std::string message, std::string file, int line, int pos);
  };
  struct parser;
  struct tokenizer;
  struct block_analyzer;
  struct type_checker;
  struct def_class_visitor;
  struct file_data {
    parser *parser_{};
    tokenizer *tokenizer_{};
    block_analyzer *block_analyzer_{};
    type_checker *type_checker_{nullptr};
    def_class_visitor *dsv_{nullptr};
  };
  enum class scanning_step {
    /** Step 0) At the begining when we scan main, or find import */
    NOTHING_DONE = 0,
    /** Step 1) Scan macro soup */
    SOUP_SCAN_DONE = 1,
    /** Step 2) Create an environment for the file's macro's to be excuted */
    ENV_CREATED = 2,
    /** Step 3) Execute macros!{} */
    MACROS_SETUP_DONE = 3,
    /** Step 4) Expand dsl!{} + parse again */
    MACROS_EXPANDED = 4,
    /** Step 5) Parse the file */
    PARSE_DONE = 5,
    /** Step 7) If all above are done, and no imports are left */
    ALL_PARSING_SUCCESSFUL = 7,
    /** If any of above fails */
    FAILURE = -999,
  };
  struct file_info {
    std::filesystem::path filepath_;
    std::string prefix_;
    file_data *data_{nullptr};
    scanning_step step_{scanning_step::NOTHING_DONE};
  };
}// namespace yaksha
#endif